# -*- coding: utf-8 -*-

import json
from time import time 
import base64
import hmac
from hashlib import sha1
from urllib import quote
import logging 

import webapp2
from google.appengine.api import urlfetch
from google.appengine.api import files

import twitterAuth # authentication information for Twitter

host = 'koreantrends' # if host is changed to 'dev', then codes in development will be executed.

def storeKoreanTweetList(koreanTweetList):
    '''
    save Korean tweets to blobstore
    '''  
    if host == 'dev':
        fileName = 'test_korean_twitter_sample'
    else:
        fileName = 'korean_twitter_sample'
        
    maxRetry = 3
    errorCount = 0
    while errorCount < maxRetry:
        try:
            blobName = files.blobstore.create(mime_type = 'text/plain', _blobinfo_uploaded_filename = fileName)
            break
        except Exception, e:
            logging.error('Failed to open ' + fileName)
            logging.error(e)
            errorCount += 1
    if errorCount == maxRetry:
        logging.error('Tried %d times to open %s, but failed' % (maxRetry, fileName))
        return

    with files.open(blobName, 'a') as f:
        for (tweetID, text) in koreanTweetList:
            f.write(str(tweetID) + '\t' + removeControlCharacters(text) + '\n')
            
    files.finalize(blobName)
    logging.debug('Saved in ' + fileName)    
    
    return

def removeControlCharacters(text):
    '''
    remove some control characters.
    '''
    for controlCharacter in ['\n', '\r', '\t']:
        text = text.replace(controlCharacter, ' ')
       
    return text

def parseToStore(twitterSample):
    '''
    parse Twitter sample data and save Korean tweets only
    '''
    countAll = 0
    countKorean = 0
    koreanTweetList = []
    for tweet in twitterSample.content.split('\n'):
        try:
            if tweet != '': tweet = json.loads(tweet)
            else: continue
        except ValueError:
            logging.warning('JSON ValueError: ' + tweet)
            continue

        if tweet.has_key('text'):
            countAll += 1
            for char in tweet['text']:
                if char >= u'\uAC00' and char <= u'\uD7AF': # unicode range of Korean
                    countKorean += 1
                    koreanTweetList.append((tweet['id'], tweet['text']))
                    break
    
    storeKoreanTweetList(koreanTweetList)
    logging.debug('All tweets: ' + str(countAll) + ', Korean tweets: ' + str(countKorean))
    
    return

def generateNonce():
    '''
    generate 32byte random string for nonce in OAuth
    '''
    candidates = base64.encodestring(str(time()) + str(time()) + str(time()))
    if candidates.isalnum():
        nonce = candidates[:32]
    else:
        nonce = ""
        for char in candidates:
            if char.isalnum(): nonce += char
            if len(nonce) == 32: break
    
    return nonce

def generateSignature(signingKey, url, consumerKey, nonce, signatureMethod, timestamp, token, version):
    '''
    generate signature for Twitter OAuth
    '''
    # The keys of parameter should be sorted alphabetically
    parameters = 'oauth_consumer_key=' + consumerKey + '&'
    parameters = parameters + 'oauth_nonce=' + nonce + '&'
    parameters = parameters + 'oauth_signature_method=HMAC-SHA1&'
    parameters = parameters + 'oauth_timestamp=' + str(timestamp) + '&'
    parameters = parameters + 'oauth_token=' + token + '&'
    parameters = parameters + 'oauth_version=1.0'
    
    baseString = 'GET&' + quote(url, safe='') + '&' + quote(parameters, safe='')
    
    return base64.b64encode(hmac.new(signingKey, baseString, sha1).digest())
    
def openURL():
    '''
    open Twitter sample API
    '''
    url = 'https://stream.twitter.com/1.1/statuses/sample.json'
    
    if host == 'dev':
        consumerKey = twitterAuth.testConsumerKey
        token = twitterAuth.testAccessToken
        signingKey = quote(twitterAuth.testConsumerSecret, safe='') + '&' + quote(twitterAuth.testTokenSecret, safe='')
    else:
        consumerKey = twitterAuth.consumerKey
        token = twitterAuth.accessToken
        signingKey = quote(twitterAuth.consumerSecret, safe='') + '&' + quote(twitterAuth.tokenSecret, safe='')
            
    nonce = generateNonce()
    signatureMethod = 'HMAC-SHA1'
    timestamp = int(time())
    version = '1.0'
 
    signature = generateSignature(signingKey, url, consumerKey, nonce, signatureMethod, timestamp, token, version)
    
    authString = 'OAuth '
    authString = authString + 'oauth_consumer_key="' + consumerKey + '", '
    authString = authString + 'oauth_nonce="' + nonce + '", '
    authString = authString + 'oauth_signature="' + quote(signature, safe='') + '", '
    authString = authString + 'oauth_signature_method="' + signatureMethod + '", '
    authString = authString + 'oauth_timestamp="' + str(timestamp) + '", '
    authString = authString + 'oauth_token="' + token + '", '
    authString = authString + 'oauth_version="' + version +'"'

    # Though deadline is set to 360,
    # the connection will continue just for 300 seconds
    # because another instance will try to connect Twitter sample API which allow just one connection a account.
    return urlfetch.fetch(url, headers={'Authorization' : authString}, deadline = 360)

def getTwitterSampleData():
    '''
    get Twitter sample data into buffer by google's urlfetch 
    because urlfetch stores all response data into buffer until the connection is closed.
    '''
    maxRetry = 3
    retry = 0
    while retry < maxRetry:
        try:
            twitterSample = openURL()
            break
        except urlfetch.DownloadError, e: 
            logging.warning(e)
            retry += 1
    if retry == maxRetry:
        logging.error('Tried %d times but failed.' % maxRetry)
        return None
        
    if twitterSample.status_code != 200:
        logging.warning('Twitter sample API error: ' + str(twitterSample.status_code))
    else:
        logging.debug('Length of downloaded sample: ' + str(len(twitterSample.content)))

    return twitterSample

def startToFetch():
    '''
    start to fetch Twitter sample data
    '''
    twitterSample = getTwitterSampleData()
    if twitterSample and twitterSample.status_code == 200:
        parseToStore(twitterSample)

    return

class FetchKoreanTwitterSample(webapp2.RequestHandler):
    '''
    fetch Twitter sample data by Twitter sample API, and parse to store it in BlobStore
    '''
    def get(self):
        self.response.headers['Content-Type'] = 'text/plain'
        
        fullDomain = self.request.headers['Host']
        if fullDomain.find('dev.') != -1 or fullDomain.find('localhost:') != -1:
            global host
            host = 'dev'
            logging.debug('Host: ' + host)
                
        # When called, there is a parameter 'number' whose value is 0 or 1 in order to make two instances. 
        startToFetch()
        
        return
        
app = webapp2.WSGIApplication([('/fetch_korean_twitter_sample', FetchKoreanTwitterSample)], debug = True)
